
# calendar
from datetime import date
from itertools import groupby

from django.utils.html import conditional_escape as esc
from django.utils.encoding import smart_str, smart_unicode

from mysite.tj3.models import *
from mysite.tj3.utility import *
from mysite.settings import *
import os, sys, shutil, re
import os.path
from subprocess import Popen, PIPE
import csv
from django.core import serializers
from string import capwords
from datetime import datetime
from utility import CleanName

DEFAULT_TASK_COLUMNS=['bsi','id','scenario','start','end','duration', 'complete','status','flags']
DEFAULT_RESOURCE_COLUMNS=['bsi','id','start','end','effort','effortdone','effortleft','complete']

def CompileProject(proj, scenario, task_reports, resource_reports):
	status_csv=[]
	load_csv=[]
	customer_reports=[]
	dest=proj.working_path

	# step 1: set up project
	tjp='auto_plan.tjp'
	task_tji,resource_tji,journal_tji, report_tji=ProjectDumpToFile(proj,scenario, dest,tjp)

	# step 2: status CSV
	reports=[]
	for s in [proj.scenario, scenario]:
		id, report =DumpTaskReport(proj, s.name,
                            format='csv',
                            start=proj.start,
                            end=proj.end,
                            # this order is siginficant!! It has to match status parse function!
                            cols='id,start,end,duration,complete,effort,effortdone,effortleft',)
		reports.append(report)
		status_csv.append((id+'.csv',s))
                            
	# step 2: resource load CSV
	for s in [proj.scenario, scenario]:
		id, report =DumpResourceReport(proj, s.name,
                            format='csv',
							start=proj.start,
							end=proj.end,
                            cols='id',
                            resolution='daily',)
		reports.append(report)
		load_csv.append((id+'.csv',s))

	# step 2: dump status report
	for r in task_reports:
		if len(r.tags) and ',' in r.tags: tags=r.tags.split(',')
		else: tags=[]
		id, report=DumpTaskReport(proj, scenario=r.scenario.name,
								compare_to=[a.name for a in r.compare_to.all()],
								format=r.format,
								start=r.start,
								end=r.end,
								cols=r.cols,
								status=r.completion,
								flags=tags,
								milestone_only=r.milestone_only)
		reports.append(report)
		customer_reports.append((id,r))
		
	for r in resource_reports:
		if len(r.tags) and ',' in r.tags: tags=r.tags.split(',')
		else: tags=[]
		id, report=DumpResourceReport(proj, scenario=r.scenario.name,
								resource=r.resource,
								compare_to=[a.name for a in r.compare_to.all()],
								format=r.format,
								start=r.start,
								end=r.end,
								cols=r.cols,
								resolution=r.resolution,
								flags=tags,)
		reports.append(report)
		customer_reports.append((id,r))
	
	# Dump all reports inito a TJI
	open(os.path.join(dest,report_tji),'w').write('\n'.join(reports))	
	
	# step 3: call tj3
	# Note: "dest" determines where the reports will be on your file system.
	cmd=['tj3','-o', dest,os.path.join(dest,tjp)]
	p = Popen(cmd, stdout=PIPE, stderr=PIPE)
	stdout, stderr = p.communicate()
	
	# In case of an error, no need to proceed further.
	# Calling these two functions will purge the existing error records!
	# So make sure you only call this once!
	try:
		has_error = ParseStderr(proj, stderr)
		error_parsed = True
	except: # if the compiler bombed, we catch it here.
		has_error = True
		error_parsed = False

	if has_error:
		# In case of an error, reports won't be complete, so there is "None" reports generated.
		# In reality, there might be some reports already generated before the TJ3 bombed.
		# But for precaution, just consider nothing is valid.
		return (tjp,task_tji,resource_tji,journal_tji,report_tji,None,stdout,stderr, error_parsed)
			
	# Update task status
	for a,scenario in status_csv:
		# Purge all status.
		# Task status can change even it was marked as completed before.
		# So better purge all.
		TaskStatus.objects.filter(project=proj,scenario=scenario).delete()
		
		# import new status
		ParseTaskStatus(open(os.path.join(dest,a),'rb'),proj, scenario)
		
	# Update resource load
	for a,scenario in load_csv:
		# Purge resoruce loads that are from TODAY and BEYOND!
		# Historical records should not be touched, ever!!
		ResourceLoad.objects.filter(project=proj,scenario=scenario,when__gte=date.today()).delete()
		
		ParseTimesheet(open(os.path.join(dest,a),'rb'),proj, scenario)

	# Read back customer reports
	for id,report_instance in customer_reports:
		tj_report, created=TJReport.objects.get_or_create(project=proj,template=report_instance)
		
		# Read generated report data
		format = report_instance.format
		raw_data = open(os.path.join(dest,'%s.%s' % (id,format))).read()
		if format=='html': 
			raw_data = ExtractReportTable(raw_data)
		tj_report.report_data=re.sub('\s+',' ',raw_data)
		tj_report.save()
	return (tjp,task_tji,resource_tji,journal_tji,report_tji,customer_reports,stdout, stderr,error_parsed)

def ParseStderr (proj, stderr):
	# Purge all error records for this project
	TJError.objects.filter(project=proj).delete()
	
	# All possible scenarios
	scenarios=proj.scenario.get_descendants(include_self=True)
	
	# Parse stderr
	has_error=False
	pat=re.compile('(?<=Task|task)\s(?P<task>\S+)')
	for a in stderr.split(proj.working_path):
		if not len(a): continue
		
		tjerror=TJError()
		tjerror.project=proj
		tokens=[b.strip() for b in a.split(':')]
	
		# token[0] is the file name
		# token[1] is the lineno
		tjerror.filename=tokens[0].strip()
		tjerror.lineno=tokens[1]
	
		# token[2]
		# Get scenarios. There are multiple cases to consider:
		# Case 1: tokens[2] will be in format "Error in scenario plan"
		# or "Warning in scenario plan".
		# Case 2: tokens[2] has only "Error" or "Warning". Embedded in the 
		# actual msg will be "actual: duration", or "plan: duration".	
		if 'Error' in tokens[2]: has_error=True
		temp=tokens[2].split(' ')
		tjerror.type=temp[0].lower()
		try:
			tjerror.scenario=scenarios.get(name=temp[-1])
		except: pass
			
		# After this token, everything is a message
		msg=':'.join(tokens[3:])
		
		# Look up task		
		temp=pat.search(msg)
		if temp:
			task_full_path=temp.group('task')
			tjerror.task=Task.objects.get(full_path=task_full_path)
		
			# Last, save the msg
			tjerror.msg= msg.replace(task_full_path,'')
		else:
			tjerror.msg=msg
		
		# Save
		tjerror.save()
		
	return has_error
	
def DumpTaskReport (proj, scenario, cols=DEFAULT_TASK_COLUMNS, compare_to=[], format='html', start=date.today(), end=date.today(), status='all', flags=[], milestone_only=False):
		
	# filter by task flags
	filters = ['1']
	if len(flags): filters+=flags
	
	# milestones
	milestone={True:'milestone', False:''}
	if milestone_only: filters.append('ismilestone(%s)' % scenario)
		
	# completion
	filter_completed='(%s.end<=${now})' % scenario
	filter_in_progress='((%s.start <= ${now}) & (${now} < %s.end))' %(scenario, scenario)
	filter_not_started='(%s.start>${now})' % scenario
	completion={'all':'1', 'completed':filter_completed, 'in progress': filter_in_progress, 'not started': filter_not_started}
	filters.append(completion[status])	
	
	# time band
	filters.append('isongoing(%s)'% scenario)
				
	# we customize:
	# report id
	# report file name
	# headline
	# filter (hidetask)
	report_body=''
	#if len(past): report_body+='start %%{${now}-%s}\n' % past
	#if len(future): report_body+='end %%{${now}+%s}\n' % future
	report_body+='start %s\n' % str(start)
	report_body+='end %s\n' % str(end)
	report_body += '''formats %s
					scenarios %s 
					selfcontained yes
					sorttasks tree, %s.start.up
					timeformat "%s"
					loadunit days 
					columns %s
					''' % (format, ','.join(compare_to+[scenario]),
							scenario, # sort by children scenario.start.up
							proj.timeformat,cols,)
	
	# replace multiple whitespaces with single white space
	# then replace that with an underscore
	# then turn everything to upper case
	headline = [proj.name, 'V1', scenario, ' '.join(compare_to), 'taskreport', str(start), str(end), ' '.join(flags), milestone[milestone_only], status]
	id = re.sub('\s+',' ',' '.join(headline)).replace(' ','_').upper()
	headline = ' - '.join(headline).upper()
		
	# report filename will be "id".format
	report = '''taskreport "%s" {
				%s # report body
				headline "%s"	
				hidetask ~(%s)
				}
				''' % (id, report_body, headline, ' & '.join(filters))
				
	# Reformat the text string. There are too many indents due to RE string used above.
	aa=[a.strip() for a in report.strip().split('\n')]
	bb=['\t'+b for b in aa[1:-1]]
	report='\n'.join([aa[0]]+bb+[aa[-1]])
	return (id, re.sub('\t+','\t',report))

def DumpResourceReport (proj, scenario, cols=DEFAULT_RESOURCE_COLUMNS, compare_to=[], format='html', start=date.today(),end=date.today(), flags=[], resolution='daily', resource=''):
		
	# filter by resource flags
	filters = ['1']
	if len(flags): filters+=flags
	
	# filter by resource name
	if resource and len(resource): filters.append('(%s.id="%s")'%(scenario,resource))
				
	# we customize:
	# report id
	# report file name
	# headline
	# filter (hidetask)
	#report_body = '''start ${now}
	#if len(from_now): report_body = 'start ${now}\nend %%{${now}+%s}' % from_now
	#else: report_body=''
	report_body='start %s\n' % str(start)
	report_body+='end %s\n' % str(end)
	report_body += '''formats %s
					scenarios %s
					sorttasks %s.start.up
					selfcontained yes
					timeformat "%s"
					loadunit shortauto
					columns %s
					''' % (format, ','.join([scenario]+compare_to),
							scenario, # sort by children scenario.start.up
							proj.timeformat, ','.join([cols,resolution]),)
	
	# replace multiple whitespaces with single white space
	# then replace that with an underscore
	# then turn everything to upper case
	headline = [proj.name, 'V1', scenario, ' '.join(compare_to), 'resourcereport', str(start), str(end), ' '.join(flags), resolution]

	# id will be the filename that TJ3 creates for this report.
	id = re.sub('\s+',' ',' '.join(headline)).replace(' ','_').upper()

	# will be the table caption TJ3 uses.
	headline = ' - '.join(headline).upper()
		
	# report filename will be "id".format
	report = '''resourcereport "%s" {
				%s # report body
				headline "%s"				
				hideresource ~(%s)
				hidetask ~(isleaf() & isongoing(%s))					
				}
				''' % (id, report_body, headline, ' & '.join(filters), scenario)
	# Reformat the text string. There are too many indents due to RE string used above.
	aa=[a.strip() for a in report.strip().split('\n')]
	bb=['\t'+b for b in aa[1:-1]]
	report='\n'.join([aa[0]]+bb+[aa[-1]])
	return (id,re.sub('\t+','\t',report))


# Extrat table from TJ reports, since we are generating one report at a time
# I want to read them back and put reports together as one HTML page
from HTMLParser import HTMLParser

class TJReportTableExtrator(HTMLParser):
	def __init__(self):
		self.reset() 
		self.start_lineno=0
		self.end_lineno=0
	
	def handle_starttag(self,tag,attrs):
		if tag == 'table':
			for attr,val in attrs:
				if attr=='class' and val=='tj_table_frame':
					(self.start_lineno, pos) =self.getpos()
		
		if tag == 'div':
			for attr,val in attrs:
				if attr=='class' and val=='copyright':
					(self.end_lineno,pos) =self.getpos()
					print self.end_lineno

def ExtractReportTable(data):
	b=TJReportTableExtrator()
	b.feed(data)
	return ''.join(data.split('\n')[b.start_lineno-1:b.end_lineno-1])

def ParseTaskStatus (file, proj, scenario):
	# IMPORTANT!!!! -- CSV is in format: ID, start, end, duration, effort, completion
	rownum = 0
	for r in csv.reader(file, delimiter=';', quotechar='"'):
		# remember the headerline, it has the dates
		if rownum == 0: 
			rownum += 1
			continue

		task = Task.objects.filter(full_path=r[0])[0]
		start = date.fromtimestamp(mktime(strptime(r[1], task.project.timeformat)))
		end = date.fromtimestamp(mktime(strptime(r[2], task.project.timeformat)))
		duration=float(r[3])
		
		# completion is "99%", so we strip off the '%' and convert it to int
		completion=int(r[4][:-1])
		
		# effort, effortdone, effortleft
		effort=float(r[5])
		effortdone=float(r[6])
		effortleft=float(r[7])
		
		# look up the instance
		temp, created=TaskStatus.objects.get_or_create(task=task,scenario=scenario, project=proj)
		temp.start=start
		temp.end=end
		temp.duration=duration
		temp.effort=effort
		temp.effortdone=effortdone
		temp.effortleft=effortleft
		temp.completion=completion
		temp.save()
	
def ParseTimesheet (file, proj, scenario):
	# Purge all timesheet
	ResourceLoad.objects.filter(project=proj,scenario=scenario).delete()
	
	# parse timesheet.csv
	# IMPORTANT!!!! -- CSV is in format: ID, work load on a day -> workload will be in hours!
	workload={}
	header=[]
	rownum = 0
	current_resource = None

	for r in csv.reader(file, delimiter=';', quotechar='"'):
		# remember the headerline, it has the dates
		if rownum == 0: 
			header=r
			rownum += 1
			continue

		# look up task
		tasks = Task.objects.filter(full_path=r[0])
		if len(tasks): 
			task=tasks[0]
			if not task.is_leaf_node(): continue #only leaf task is meaningful
			
		else: # try if this is a resource, this to update current_resource
			first, last, worklimit = tuple(r[0].split('_'))
			resources = Resource.objects.filter(first_name=first,last_name=last,work_limit=int(worklimit))
			if len(resources): 
				current_resource=resources[0]
				# add to the workload dict
				if current_resource not in workload:
					workload[current_resource]={}
			continue # if not a task line, we do not need to look further!

		# search for workload
		# IMPORTANT!: this is assuming that the 2nd column and onward in the CSV will be dates!
		for i in range(1,len(header)):
			assignment_day = header[i]
			load=r[i]
			if re.search('(?P<load>[\d.]*)[hHdD]', load):
				if assignment_day not in workload[current_resource]: workload[current_resource][assignment_day]=[]
				workload[current_resource][assignment_day].append((task.full_path,load))
				
				# at this point, create resource load instance
				when = date.fromtimestamp(mktime(strptime(assignment_day, task.project.timeformat)))
				temp, created=ResourceLoad.objects.get_or_create(task=task, resource=current_resource, when=when, scenario=scenario, project=proj)
				temp_load=float(load[:-1])
				if load[-1] in ['d','D']: # if the time is in Day, we need to convert them into hours
					temp_load= temp_load*8 # this is hardcoded for now.
				temp.load= temp_load
				temp.save()
	return workload	
